import os
import logging
import requests
from urllib.parse import urlparse
from scrapy.exceptions import DropItem

class CustomImagePipeline:
    def __init__(self, store_dir):
        self.store_dir = store_dir
        self.logger = logging.getLogger("CustomImagePipeline")
        os.makedirs(self.store_dir, exist_ok=True)  # 确保存储目录存在

    @classmethod
    def from_crawler(cls, crawler):
        # 从settings获取存储路径
        return cls(store_dir=crawler.settings.get("IMAGES_STORE", "custom_images"))

    def process_item(self, item, spider):
        if not item.get("image_urls"):
            self.logger.warning("Item中无image_urls字段，跳过处理")
            return item

        # 遍历所有URL并下载
        for idx, url in enumerate(item["image_urls"]):
            try:
                # 生成文件名（基于URL哈希，避免重复）
                parsed_url = urlparse(url)
                filename = f"image_{idx}_{hash(url)}.jpg"  # 哈希值确保文件名唯一
                save_path = os.path.join(self.store_dir, filename)

                # 发送请求（模拟浏览器头信息）
                headers = {
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
                    "Referer": "https://image.baidu.com",  # 关键：模拟从百度图片页跳转
                    "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
                }

                self.logger.info(f"开始下载: {url} -> {save_path}")
                response = requests.get(
                    url,
                    headers=headers,
                    timeout=30,
                    allow_redirects=True  # 允许重定向
                )

                # 检查响应状态
                if response.status_code == 200:
                    # 验证是否为图片（简单检查响应头）
                    if "image" in response.headers.get("Content-Type", ""):
                        with open(save_path, "wb") as f:
                            f.write(response.content)
                        self.logger.info(f"下载成功: {save_path}")
                    else:
                        self.logger.error(f"非图片内容: {url} (Content-Type: {response.headers.get('Content-Type')})")
                else:
                    self.logger.error(f"下载失败（状态码: {response.status_code}）: {url}")

            except Exception as e:
                self.logger.error(f"下载异常（URL: {url}）: {str(e)}")

        return item